import scrapy
from scrapy import Selector, Request
from scrapy.http import HtmlResponse

from Division.items import DivisionItem

GRADES = {
    "PROVINCE_GRADE": 1,
    "CITY_GRADE": 2,
    "COUNTY_GRADE": 3,
    "TOWN_GRADE": 4,
    "VILLAGE_GRADE": 5
}

default_parent_code = 1


class DivisionSpider(scrapy.Spider):
    name = "division"
    allowed_domains = ["www.stats.gov.cn"]
    start_urls = ["http://www.stats.gov.cn/sj/tjbz/tjyqhdmhcxhfdm/2022/"]

    def parse(self, response: HtmlResponse, **kwargs):
        sel = Selector(response)

        list_items = sel.xpath("//tr[@class='provincetr']/td/a")
        for list_item in list_items:
            division_item = DivisionItem()  # 组装数据
            division_item["code"] = list_item.xpath("./@href").extract_first().rstrip(".html")
            division_item["parent_code"] = default_parent_code
            division_item["name"] = list_item.xpath("./text()").extract_first()
            division_item["grade"] = GRADES["PROVINCE_GRADE"]
            yield division_item  # 将组装好的数据传递给引擎

        # 获取子页面的跳转链接
        href_list = sel.xpath("//tr[@class='provincetr']/td/a/@href")
        for href in href_list:
            url = response.urljoin(href.extract())  # 对子页面URL进行拼接
            yield Request(url=url, callback=self.city_parse)

    def city_parse(self, response):
        sel = Selector(response)

        list_items = sel.xpath("//tr[@class='citytr']/td[2]/a")
        for list_item in list_items:
            division_item = DivisionItem()
            division_item["code"] = list_item.xpath("./@href").extract_first().split("/")[-1].rstrip(".html")
            division_item["parent_code"] = list_item.xpath("./@href").extract_first().split("/")[0]
            division_item["name"] = list_item.xpath("./text()").extract_first()
            division_item["grade"] = GRADES["CITY_GRADE"]
            yield division_item

        href_list = sel.xpath("//tr[@class='citytr']/td/a/@href")
        for href in href_list:
            url = response.urljoin(href.extract())
            yield Request(url=url, callback=self.county_parse)

    def county_parse(self, response):
        sel = Selector(response)

        list_items = sel.xpath("//tr[@class='countytr']/td[2]/a")
        for list_item in list_items:
            division_item = DivisionItem()
            division_item["code"] = list_item.xpath("./@href").extract_first().split("/")[-1].rstrip(".html")
            division_item["parent_code"] = list_item.xpath("./@href").extract_first().split("/")[-1].split(".")[0][:4]
            division_item["name"] = list_item.xpath("./text()").extract_first()
            division_item["grade"] = GRADES["COUNTY_GRADE"]
            yield division_item

        href_list = sel.xpath("//tr[@class='countytr']/td/a/@href")
        for href in href_list:
            url = response.urljoin(href.extract())
            yield Request(url=url, callback=self.town_parse)

    def town_parse(self, response):
        sel = Selector(response)

        list_items = sel.xpath("//tr[@class='towntr']/td[2]/a")
        for list_item in list_items:
            division_item = DivisionItem()
            division_item["code"] = list_item.xpath("./@href").extract_first().split("/")[-1].rstrip(".html")
            division_item["parent_code"] = list_item.xpath("./@href").extract_first().split("/")[-1].split(".")[0][:6]
            division_item["name"] = list_item.xpath("./text()").extract_first()
            division_item["grade"] = GRADES["TOWN_GRADE"]
            yield division_item

        href_list = sel.xpath("//tr[@class='towntr']/td/a/@href")
        for href in href_list:
            url = response.urljoin(href.extract())
            yield Request(url=url, callback=self.village_parse)

    def village_parse(self, response):
        sel = Selector(response)

        list_items = sel.xpath("//tr[@class='villagetr']")
        for list_item in list_items:
            division_item = DivisionItem()
            division_item["code"] = list_item.xpath("./td[1]/text()").extract_first()
            division_item["parent_code"] = list_item.xpath("./td[1]/text()").extract_first()[:9]
            division_item["name"] = list_item.xpath("./td[3]/text()").extract_first()
            division_item["grade"] = GRADES["VILLAGE_GRADE"]
            yield division_item


